{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from __future__ import (print_function, absolute_import)\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import keras\n",
    "import keras.backend as K\n",
    "from keras import datasets\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.models import load_model\n",
    "\n",
    "from models import CNN, VGG8\n",
    "from wide_resnet import WideResidualNetwork\n",
    "\n",
    "from keras.callbacks import (\n",
    "    ReduceLROnPlateau,\n",
    "    LearningRateScheduler,\n",
    "    CSVLogger,\n",
    "    EarlyStopping,\n",
    "    ModelCheckpoint)\n",
    "from keras_tqdm import TQDMNotebookCallback\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"1\"\n",
    "\n",
    "global _SESSION\n",
    "config = tf.ConfigProto(allow_soft_placement=True)\n",
    "config.gpu_options.allow_growth = True\n",
    "_SESSION = tf.Session(config=config)\n",
    "K.set_session(_SESSION)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CIFAR110"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_classes = 11\n",
    "batch_size = 128\n",
    "epochs = 200\n",
    "data_augmentation = True\n",
    "checkpoint = None\n",
    "# checkpoint = 'model_checkpoint_cifar110_vgg8.h5'\n",
    "title = 'cifar110_vgg8'\n",
    "# title = 'cifar110_wide_resnet'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "(x_train_10, y_train_10), (x_test_10, y_test_10) = datasets.cifar10.load_data()\n",
    "(x_train_100, y_train_100), (x_test_100, y_test_100) = datasets.cifar100.load_data()\n",
    "\n",
    "y_train_10 = y_train_10 + 1\n",
    "y_test_10 = y_test_10 + 1\n",
    "y_train_100[...] = 0\n",
    "y_test_100[...] = 0\n",
    "\n",
    "x_train = np.concatenate((x_train_10, x_train_100), axis=0).astype('float32')\n",
    "y_train = np.concatenate((y_train_10, y_train_100), axis=0).astype('int8')\n",
    "x_test = np.concatenate((x_test_10, x_test_100), axis=0).astype('float32')\n",
    "y_test = np.concatenate((y_test_10, y_test_100), axis=0).astype('int8')\n",
    "\n",
    "x_train = x_train.astype(K.floatx())\n",
    "x_test = x_test.astype(K.floatx())\n",
    "\n",
    "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = keras.utils.to_categorical(y_test, num_classes)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(100000, 32, 32, 3) (100000, 11)\n",
      "(20000, 32, 32, 3) (20000, 11)\n"
     ]
    }
   ],
   "source": [
    "print(x_train.shape, y_train.shape)\n",
    "print(x_test.shape, y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_55 (Conv2D)           (None, 32, 32, 32)        896       \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_55 (LeakyReLU)   (None, 32, 32, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_56 (Conv2D)           (None, 32, 32, 32)        9248      \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_56 (LeakyReLU)   (None, 32, 32, 32)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_28 (MaxPooling (None, 16, 16, 32)        0         \n",
      "_________________________________________________________________\n",
      "dropout_37 (Dropout)         (None, 16, 16, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_57 (Conv2D)           (None, 16, 16, 64)        18496     \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_57 (LeakyReLU)   (None, 16, 16, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_58 (Conv2D)           (None, 16, 16, 64)        36928     \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_58 (LeakyReLU)   (None, 16, 16, 64)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_29 (MaxPooling (None, 8, 8, 64)          0         \n",
      "_________________________________________________________________\n",
      "dropout_38 (Dropout)         (None, 8, 8, 64)          0         \n",
      "_________________________________________________________________\n",
      "conv2d_59 (Conv2D)           (None, 8, 8, 128)         73856     \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_59 (LeakyReLU)   (None, 8, 8, 128)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_60 (Conv2D)           (None, 8, 8, 128)         147584    \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_60 (LeakyReLU)   (None, 8, 8, 128)         0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_30 (MaxPooling (None, 4, 4, 128)         0         \n",
      "_________________________________________________________________\n",
      "dropout_39 (Dropout)         (None, 4, 4, 128)         0         \n",
      "_________________________________________________________________\n",
      "flatten_10 (Flatten)         (None, 2048)              0         \n",
      "_________________________________________________________________\n",
      "dense_19 (Dense)             (None, 512)               1049088   \n",
      "_________________________________________________________________\n",
      "activation_19 (Activation)   (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dropout_40 (Dropout)         (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_20 (Dense)             (None, 11)                5643      \n",
      "_________________________________________________________________\n",
      "activation_20 (Activation)   (None, 11)                0         \n",
      "=================================================================\n",
      "Total params: 1,341,739\n",
      "Trainable params: 1,341,739\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "if title == 'cifar110_vgg8':\n",
    "    model = VGG8(input_shape=x_train.shape[1:], num_classes=num_classes)\n",
    "elif title == 'cifar110_wide_resnet':\n",
    "    model = WideResidualNetwork(depth=28, width=8, dropout_rate=0.5,\n",
    "                                classes=num_classes, include_top=True,\n",
    "                                weights=None)\n",
    "if checkpoint is not None:\n",
    "    print(checkpoint)\n",
    "    model = load_model(checkpoint)\n",
    "    \n",
    "\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer='adam',\n",
    "              metrics=['accuracy'])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Checkpoint\n",
    "checkpointer = ModelCheckpoint(\n",
    "    filepath=\"model_checkpoint_{}.h5\".format(title),\n",
    "    verbose=1,\n",
    "    save_best_only=True)\n",
    "\n",
    "# csvlogger\n",
    "csv_logger = CSVLogger(\n",
    "    'csv_logger_{}.csv'.format(title))\n",
    "# EarlyStopping\n",
    "early_stopper = EarlyStopping(monitor='val_loss',\n",
    "                              min_delta=0.001,\n",
    "                              patience=100)\n",
    "# Reduce lr with schedule\n",
    "# def schedule(epoch):\n",
    "#     lr = K.get_value(sgd.lr)\n",
    "#     if epoch in [60, 120, 160]:\n",
    "#         lr = lr * np.sqrt(0.1)\n",
    "#     return lr\n",
    "# lr_scheduler = LearningRateScheduler(schedule)\n",
    "\n",
    "# Reduce lr on plateau\n",
    "lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1),\n",
    "                               cooldown=0,\n",
    "                               patience=20,\n",
    "                               min_lr=1e-6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using real-time data augmentation.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0669289d2e814afaa276f94c65efafbe"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f3f39821a85a413ba7f658ea1f1b33e5"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 1.3991 - acc: 0.5504Epoch 00000: val_loss improved from inf to 1.05994, saving model to model_checkpoint_cifar110_vgg8.h5\n",
      "781/781 [==============================] - 92s - loss: 1.3988 - acc: 0.5504 - val_loss: 1.0599 - val_acc: 0.6381\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2591b6f091694c63a9cfe595c50531a0"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 1.0801 - acc: 0.6347Epoch 00001: val_loss improved from 1.05994 to 0.86855, saving model to model_checkpoint_cifar110_vgg8.h5\n",
      "781/781 [==============================] - 91s - loss: 1.0798 - acc: 0.6348 - val_loss: 0.8686 - val_acc: 0.6986\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "435aff5cdc7c49e2adc2489221be7cca"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.9752 - acc: 0.6696Epoch 00002: val_loss improved from 0.86855 to 0.82663, saving model to model_checkpoint_cifar110_vgg8.h5\n",
      "781/781 [==============================] - 91s - loss: 0.9752 - acc: 0.6696 - val_loss: 0.8266 - val_acc: 0.7135\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "20a6f055f52d4b93a4ec9c1233868f56"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 4/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.9151 - acc: 0.6865Epoch 00003: val_loss improved from 0.82663 to 0.75716, saving model to model_checkpoint_cifar110_vgg8.h5\n",
      "781/781 [==============================] - 56s - loss: 0.9150 - acc: 0.6865 - val_loss: 0.7572 - val_acc: 0.7385\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "45db3df365074515a34ec5711c010f61"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 5/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.8786 - acc: 0.7003Epoch 00004: val_loss improved from 0.75716 to 0.73221, saving model to model_checkpoint_cifar110_vgg8.h5\n",
      "781/781 [==============================] - 67s - loss: 0.8786 - acc: 0.7003 - val_loss: 0.7322 - val_acc: 0.7480\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6a4ae83584414426a4964d784e89b27f"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 6/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.8508 - acc: 0.7095Epoch 00005: val_loss improved from 0.73221 to 0.69964, saving model to model_checkpoint_cifar110_vgg8.h5\n",
      "781/781 [==============================] - 92s - loss: 0.8506 - acc: 0.7096 - val_loss: 0.6996 - val_acc: 0.7548\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ba83cf974c6e443c90a5029e3bd062a7"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 7/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.8323 - acc: 0.7151Epoch 00006: val_loss did not improve\n",
      "781/781 [==============================] - 82s - loss: 0.8324 - acc: 0.7151 - val_loss: 0.7135 - val_acc: 0.7550\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c4ec14d22eba4200abbe4431685933e7"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 8/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.8121 - acc: 0.7226Epoch 00007: val_loss improved from 0.69964 to 0.67460, saving model to model_checkpoint_cifar110_vgg8.h5\n",
      "781/781 [==============================] - 90s - loss: 0.8118 - acc: 0.7227 - val_loss: 0.6746 - val_acc: 0.7660\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e443d2ac92d0491faba6f1c6d36647d5"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 9/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.8028 - acc: 0.7255Epoch 00008: val_loss improved from 0.67460 to 0.63996, saving model to model_checkpoint_cifar110_vgg8.h5\n",
      "781/781 [==============================] - 65s - loss: 0.8028 - acc: 0.7255 - val_loss: 0.6400 - val_acc: 0.7759\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "18d9f7a00ab44422a509bf3a061d6080"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 10/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7927 - acc: 0.7276Epoch 00009: val_loss did not improve\n",
      "781/781 [==============================] - 71s - loss: 0.7925 - acc: 0.7277 - val_loss: 0.6430 - val_acc: 0.7761\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "26adb07b1966427084a6b18184502fc1"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 11/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7756 - acc: 0.7333Epoch 00010: val_loss improved from 0.63996 to 0.63631, saving model to model_checkpoint_cifar110_vgg8.h5\n",
      "781/781 [==============================] - 96s - loss: 0.7758 - acc: 0.7333 - val_loss: 0.6363 - val_acc: 0.7795\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "db507017ba2541b29d80b853d4ff578d"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 12/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7708 - acc: 0.7357Epoch 00011: val_loss did not improve\n",
      "781/781 [==============================] - 95s - loss: 0.7712 - acc: 0.7356 - val_loss: 0.6467 - val_acc: 0.7774\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6c56f38910ba49e887ebc8f99813ca37"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 13/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7647 - acc: 0.7382Epoch 00012: val_loss improved from 0.63631 to 0.62464, saving model to model_checkpoint_cifar110_vgg8.h5\n",
      "781/781 [==============================] - 96s - loss: 0.7647 - acc: 0.7382 - val_loss: 0.6246 - val_acc: 0.7835\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7afa017ebdc3401bae74028e97fbf5dc"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 14/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7621 - acc: 0.7398Epoch 00013: val_loss improved from 0.62464 to 0.61555, saving model to model_checkpoint_cifar110_vgg8.h5\n",
      "781/781 [==============================] - 96s - loss: 0.7620 - acc: 0.7397 - val_loss: 0.6155 - val_acc: 0.7870\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "49023231860b4df382e93b617b3e70ec"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 15/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7536 - acc: 0.7423Epoch 00014: val_loss improved from 0.61555 to 0.60232, saving model to model_checkpoint_cifar110_vgg8.h5\n",
      "781/781 [==============================] - 81s - loss: 0.7535 - acc: 0.7423 - val_loss: 0.6023 - val_acc: 0.7918\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "91f1bb95d4564453b73f70a0f3ebbe2a"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 16/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7490 - acc: 0.7448Epoch 00015: val_loss did not improve\n",
      "781/781 [==============================] - 95s - loss: 0.7488 - acc: 0.7449 - val_loss: 0.6176 - val_acc: 0.7836\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "394e21b9e53140ba8fe72f79c483eba6"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 17/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7389 - acc: 0.7483Epoch 00016: val_loss improved from 0.60232 to 0.59799, saving model to model_checkpoint_cifar110_vgg8.h5\n",
      "781/781 [==============================] - 95s - loss: 0.7387 - acc: 0.7484 - val_loss: 0.5980 - val_acc: 0.7917\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0e093c97bb404f4e8a1b61f71d3e8148"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 18/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7355 - acc: 0.7487Epoch 00017: val_loss improved from 0.59799 to 0.58859, saving model to model_checkpoint_cifar110_vgg8.h5\n",
      "781/781 [==============================] - 86s - loss: 0.7356 - acc: 0.7487 - val_loss: 0.5886 - val_acc: 0.7967\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2f9f6a54dcaf48ccb36b4e4748111db9"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 19/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7319 - acc: 0.7494Epoch 00018: val_loss did not improve\n",
      "781/781 [==============================] - 81s - loss: 0.7319 - acc: 0.7495 - val_loss: 0.6012 - val_acc: 0.7925\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9693d5875aca4d1ab1aeb8255fd9f43a"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 20/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7362 - acc: 0.7484Epoch 00019: val_loss did not improve\n",
      "781/781 [==============================] - 83s - loss: 0.7363 - acc: 0.7484 - val_loss: 0.6026 - val_acc: 0.7954\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "de3958658219450695bd2623f3a64196"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 21/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7248 - acc: 0.7519Epoch 00020: val_loss improved from 0.58859 to 0.58207, saving model to model_checkpoint_cifar110_vgg8.h5\n",
      "781/781 [==============================] - 95s - loss: 0.7246 - acc: 0.7520 - val_loss: 0.5821 - val_acc: 0.7992\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4baeb30e822b49ecbfea720dd744f6af"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 22/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7193 - acc: 0.7547Epoch 00021: val_loss did not improve\n",
      "781/781 [==============================] - 95s - loss: 0.7192 - acc: 0.7548 - val_loss: 0.5887 - val_acc: 0.7983\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "979b61c2edc34822ae423585dc1bccd0"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 23/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7210 - acc: 0.7538Epoch 00022: val_loss did not improve\n",
      "781/781 [==============================] - 79s - loss: 0.7210 - acc: 0.7537 - val_loss: 0.6172 - val_acc: 0.7909\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "78fa692d4557481a8479521f36c66a6c"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 24/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7161 - acc: 0.7553Epoch 00023: val_loss improved from 0.58207 to 0.57664, saving model to model_checkpoint_cifar110_vgg8.h5\n",
      "781/781 [==============================] - 79s - loss: 0.7160 - acc: 0.7553 - val_loss: 0.5766 - val_acc: 0.8022\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5aeab70700df47988c4f5e3ccae1666d"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 25/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7148 - acc: 0.7543Epoch 00024: val_loss improved from 0.57664 to 0.56926, saving model to model_checkpoint_cifar110_vgg8.h5\n",
      "781/781 [==============================] - 94s - loss: 0.7149 - acc: 0.7543 - val_loss: 0.5693 - val_acc: 0.8068\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f9712ccd392841a1a6e7ed1c0d47dc2c"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 26/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7144 - acc: 0.7572Epoch 00025: val_loss improved from 0.56926 to 0.55775, saving model to model_checkpoint_cifar110_vgg8.h5\n",
      "781/781 [==============================] - 95s - loss: 0.7144 - acc: 0.7573 - val_loss: 0.5577 - val_acc: 0.8102\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "382f84700c58481ba761caf56f63bc31"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 27/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7081 - acc: 0.7588Epoch 00026: val_loss improved from 0.55775 to 0.55709, saving model to model_checkpoint_cifar110_vgg8.h5\n",
      "781/781 [==============================] - 75s - loss: 0.7080 - acc: 0.7588 - val_loss: 0.5571 - val_acc: 0.8104\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "abc599f9811d4bc29885f46d2764a899"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 28/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7090 - acc: 0.7569Epoch 00027: val_loss did not improve\n",
      "781/781 [==============================] - 95s - loss: 0.7089 - acc: 0.7569 - val_loss: 0.5706 - val_acc: 0.8058\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3d13732d014c40dc9ea1f386a9d73ac2"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 29/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7038 - acc: 0.7596Epoch 00028: val_loss did not improve\n",
      "781/781 [==============================] - 95s - loss: 0.7038 - acc: 0.7596 - val_loss: 0.5586 - val_acc: 0.8075\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "06a72b238b4449abbc9a13cceeb0c5d4"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 30/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7065 - acc: 0.7588Epoch 00029: val_loss did not improve\n",
      "781/781 [==============================] - 96s - loss: 0.7064 - acc: 0.7589 - val_loss: 0.5660 - val_acc: 0.8087\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e936086683ff4ed6ac41bf8c9d0fe1bd"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 31/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7096 - acc: 0.7588Epoch 00030: val_loss improved from 0.55709 to 0.55517, saving model to model_checkpoint_cifar110_vgg8.h5\n",
      "781/781 [==============================] - 95s - loss: 0.7095 - acc: 0.7589 - val_loss: 0.5552 - val_acc: 0.8127\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "124750aaedc84ea08eb04f7d942c0e8a"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 32/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7023 - acc: 0.7615Epoch 00031: val_loss did not improve\n",
      "781/781 [==============================] - 94s - loss: 0.7022 - acc: 0.7615 - val_loss: 0.5886 - val_acc: 0.8018\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f75eeb6c6b1643db9d4cd5e6481cc96e"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 33/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6962 - acc: 0.7631Epoch 00032: val_loss did not improve\n",
      "781/781 [==============================] - 78s - loss: 0.6961 - acc: 0.7631 - val_loss: 0.5704 - val_acc: 0.8086\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5e3423e8fba44efcbe730631080e021c"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 34/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7052 - acc: 0.7595Epoch 00033: val_loss did not improve\n",
      "781/781 [==============================] - 93s - loss: 0.7050 - acc: 0.7596 - val_loss: 0.5627 - val_acc: 0.8102\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "08d3dbb749f44e659aa6fbf2fb30611e"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 35/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6958 - acc: 0.7630Epoch 00034: val_loss did not improve\n",
      "781/781 [==============================] - 94s - loss: 0.6959 - acc: 0.7630 - val_loss: 0.5695 - val_acc: 0.8069\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "462aa67f6bce4a6b955f19445ab1461e"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 36/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7023 - acc: 0.7622Epoch 00035: val_loss did not improve\n",
      "781/781 [==============================] - 93s - loss: 0.7024 - acc: 0.7621 - val_loss: 0.5588 - val_acc: 0.8124\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0a0fec204c72401e9a1ba5edaf7831c5"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 37/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6961 - acc: 0.7639Epoch 00036: val_loss improved from 0.55517 to 0.55294, saving model to model_checkpoint_cifar110_vgg8.h5\n",
      "781/781 [==============================] - 94s - loss: 0.6961 - acc: 0.7638 - val_loss: 0.5529 - val_acc: 0.8092\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "98434f11f5be4550a2aadcdb6dacf4a0"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 38/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6946 - acc: 0.7642Epoch 00037: val_loss did not improve\n",
      "781/781 [==============================] - 93s - loss: 0.6944 - acc: 0.7642 - val_loss: 0.5581 - val_acc: 0.8063\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "848452d5b20f4a64bf802ed71b50a098"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 39/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6951 - acc: 0.7666Epoch 00038: val_loss did not improve\n",
      "781/781 [==============================] - 93s - loss: 0.6952 - acc: 0.7666 - val_loss: 0.5660 - val_acc: 0.8057\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "73bcb94ba0ba4fe789e894fb29a5ee21"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 40/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6924 - acc: 0.7642Epoch 00039: val_loss did not improve\n",
      "781/781 [==============================] - 94s - loss: 0.6924 - acc: 0.7641 - val_loss: 0.5706 - val_acc: 0.8033\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "01382e6b432d43ee8f3a50fab93a1495"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 41/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6966 - acc: 0.7634Epoch 00040: val_loss improved from 0.55294 to 0.54097, saving model to model_checkpoint_cifar110_vgg8.h5\n",
      "781/781 [==============================] - 94s - loss: 0.6966 - acc: 0.7634 - val_loss: 0.5410 - val_acc: 0.8169\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "79ee8283f8234188a0a9e19109ebb3f2"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 42/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6875 - acc: 0.7660Epoch 00041: val_loss did not improve\n",
      "781/781 [==============================] - 93s - loss: 0.6874 - acc: 0.7659 - val_loss: 0.5712 - val_acc: 0.8027\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dd817d5b499343a7a9317c9111ebaf2e"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 43/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6851 - acc: 0.7657Epoch 00042: val_loss did not improve\n",
      "781/781 [==============================] - 94s - loss: 0.6853 - acc: 0.7657 - val_loss: 0.5452 - val_acc: 0.8116\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0e26b58f166f45c3823a6091d22050c3"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 44/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6854 - acc: 0.7670Epoch 00043: val_loss did not improve\n",
      "781/781 [==============================] - 94s - loss: 0.6855 - acc: 0.7670 - val_loss: 0.5634 - val_acc: 0.8101\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ee1654ed5deb42afb2137bbe747b6f6b"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 45/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6841 - acc: 0.7677Epoch 00044: val_loss improved from 0.54097 to 0.53894, saving model to model_checkpoint_cifar110_vgg8.h5\n",
      "781/781 [==============================] - 94s - loss: 0.6842 - acc: 0.7678 - val_loss: 0.5389 - val_acc: 0.8146\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "94dbd2aee4664a8792c70b8c94225460"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 46/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6830 - acc: 0.7680Epoch 00045: val_loss did not improve\n",
      "781/781 [==============================] - 94s - loss: 0.6828 - acc: 0.7680 - val_loss: 0.5464 - val_acc: 0.8141\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f3229dd274704b35adbc9a31cd83a1c8"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 47/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6875 - acc: 0.7677Epoch 00046: val_loss did not improve\n",
      "781/781 [==============================] - 94s - loss: 0.6876 - acc: 0.7676 - val_loss: 0.5658 - val_acc: 0.8066\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c2511282a83e4822b1e82e5b79e4daf9"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 48/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6843 - acc: 0.7677Epoch 00047: val_loss improved from 0.53894 to 0.53700, saving model to model_checkpoint_cifar110_vgg8.h5\n",
      "781/781 [==============================] - 94s - loss: 0.6842 - acc: 0.7677 - val_loss: 0.5370 - val_acc: 0.8168\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f25d72700ccb44d4a7dae6ecf9481419"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 49/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6858 - acc: 0.7675Epoch 00048: val_loss improved from 0.53700 to 0.53161, saving model to model_checkpoint_cifar110_vgg8.h5\n",
      "781/781 [==============================] - 94s - loss: 0.6858 - acc: 0.7676 - val_loss: 0.5316 - val_acc: 0.8179\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "247f01409ea94f22ae5f657ec8c6c5f7"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 50/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6882 - acc: 0.7663Epoch 00049: val_loss did not improve\n",
      "781/781 [==============================] - 94s - loss: 0.6885 - acc: 0.7662 - val_loss: 0.5490 - val_acc: 0.8108\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "41d5bcf877ae4b2a8f695c7e99f16f20"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 51/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6854 - acc: 0.7686Epoch 00050: val_loss did not improve\n",
      "781/781 [==============================] - 94s - loss: 0.6854 - acc: 0.7686 - val_loss: 0.5497 - val_acc: 0.8104\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6aa88ebef8b64f92a66b45c6811ebfb6"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 52/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6811 - acc: 0.7711Epoch 00051: val_loss did not improve\n",
      "781/781 [==============================] - 94s - loss: 0.6812 - acc: 0.7711 - val_loss: 0.5430 - val_acc: 0.8197\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6411cb1963d8430e81a940c3f63b5efc"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 53/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6787 - acc: 0.7715Epoch 00052: val_loss did not improve\n",
      "781/781 [==============================] - 94s - loss: 0.6786 - acc: 0.7715 - val_loss: 0.5403 - val_acc: 0.8168\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aa8135dfdbd34156add1434929e73097"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 54/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6800 - acc: 0.7710Epoch 00053: val_loss did not improve\n",
      "781/781 [==============================] - 94s - loss: 0.6798 - acc: 0.7711 - val_loss: 0.5321 - val_acc: 0.8206\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1b4edaf97c44405b8adcd490e8562904"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 55/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6771 - acc: 0.7717Epoch 00054: val_loss did not improve\n",
      "781/781 [==============================] - 94s - loss: 0.6770 - acc: 0.7717 - val_loss: 0.5360 - val_acc: 0.8193\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0d5d5ed7476d447fa92c5d993a585b89"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 56/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6811 - acc: 0.7692Epoch 00055: val_loss did not improve\n",
      "781/781 [==============================] - 94s - loss: 0.6811 - acc: 0.7692 - val_loss: 0.5646 - val_acc: 0.8092\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "33fa670f2b3c4e6091cf4ac55a98e4f1"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 57/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6820 - acc: 0.7685Epoch 00056: val_loss did not improve\n",
      "781/781 [==============================] - 93s - loss: 0.6822 - acc: 0.7685 - val_loss: 0.5442 - val_acc: 0.8114\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cbf0f7710ce241f9980d332860d1fe8c"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 58/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6828 - acc: 0.7700Epoch 00057: val_loss did not improve\n",
      "781/781 [==============================] - 94s - loss: 0.6825 - acc: 0.7700 - val_loss: 0.5700 - val_acc: 0.8067\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cc063b33db234b628a8bd77660bcfa9a"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 59/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6782 - acc: 0.7729Epoch 00058: val_loss improved from 0.53161 to 0.52103, saving model to model_checkpoint_cifar110_vgg8.h5\n",
      "781/781 [==============================] - 94s - loss: 0.6784 - acc: 0.7728 - val_loss: 0.5210 - val_acc: 0.8238\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f85b2cea9ec141ccb0054f806a7a846b"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 60/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6782 - acc: 0.7717Epoch 00059: val_loss did not improve\n",
      "781/781 [==============================] - 94s - loss: 0.6781 - acc: 0.7717 - val_loss: 0.5297 - val_acc: 0.8184\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dd65e2ef06f3453aaa7451a28f5c3151"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 61/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6793 - acc: 0.7699Epoch 00060: val_loss did not improve\n",
      "781/781 [==============================] - 94s - loss: 0.6793 - acc: 0.7699 - val_loss: 0.5575 - val_acc: 0.8154\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "154b0e77ac4d49b9add0aa7489ca1177"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 62/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6773 - acc: 0.7718Epoch 00061: val_loss did not improve\n",
      "781/781 [==============================] - 94s - loss: 0.6773 - acc: 0.7718 - val_loss: 0.5383 - val_acc: 0.8204\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2ea30215d492456cb361cc2b0463f71c"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 63/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6780 - acc: 0.7725Epoch 00062: val_loss did not improve\n",
      "781/781 [==============================] - 94s - loss: 0.6780 - acc: 0.7725 - val_loss: 0.5481 - val_acc: 0.8141\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5b1a04ae171c454f81cf58b0dbd4dd34"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 64/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6736 - acc: 0.7721Epoch 00063: val_loss did not improve\n",
      "781/781 [==============================] - 94s - loss: 0.6737 - acc: 0.7721 - val_loss: 0.5341 - val_acc: 0.8238\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "652bd1e2bb074b658ef14bddaa689576"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 65/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6784 - acc: 0.7721Epoch 00064: val_loss did not improve\n",
      "781/781 [==============================] - 94s - loss: 0.6784 - acc: 0.7721 - val_loss: 0.5438 - val_acc: 0.8158\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2844236bb04044f6a85b08fa2cd9091b"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 66/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6671 - acc: 0.7750Epoch 00065: val_loss did not improve\n",
      "781/781 [==============================] - 94s - loss: 0.6672 - acc: 0.7750 - val_loss: 0.5336 - val_acc: 0.8142\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "daec9174141b46c59890ca54cddbd0fe"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 67/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6807 - acc: 0.7711Epoch 00066: val_loss did not improve\n",
      "781/781 [==============================] - 94s - loss: 0.6807 - acc: 0.7711 - val_loss: 0.5485 - val_acc: 0.8161\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9e71ff1b369f4ecd964cd67d2d92ecf6"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 68/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6739 - acc: 0.7728Epoch 00067: val_loss did not improve\n",
      "781/781 [==============================] - 94s - loss: 0.6739 - acc: 0.7728 - val_loss: 0.5414 - val_acc: 0.8205\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "27ac92fade954da7b294f5cf13d0fcab"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 69/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6685 - acc: 0.7743Epoch 00068: val_loss did not improve\n",
      "781/781 [==============================] - 95s - loss: 0.6686 - acc: 0.7743 - val_loss: 0.5337 - val_acc: 0.8236\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "364014521eea4d088ed70f23d6283670"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 70/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6743 - acc: 0.7742Epoch 00069: val_loss did not improve\n",
      "781/781 [==============================] - 94s - loss: 0.6742 - acc: 0.7742 - val_loss: 0.5262 - val_acc: 0.8216\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6daee89615874ae3823f58a632e1627d"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 71/200\n",
      "505/781 [==================>...........] - ETA: 30s - loss: 0.6798 - acc: 0.7724"
     ]
    }
   ],
   "source": [
    "def normalize(x):\n",
    "    \"\"\"Substract mean and Divide by std.\"\"\"\n",
    "    x -= np.array([125.3, 123.0, 113.9], dtype=K.floatx())\n",
    "    x /= np.array([63.0, 62.1, 66.7], dtype=K.floatx())\n",
    "    return x\n",
    "\n",
    "\n",
    "x_train = normalize(x_train)\n",
    "x_test = normalize(x_test)\n",
    "\n",
    "if not data_augmentation:\n",
    "    print('No data augmentation applied.')\n",
    "    model.fit(x_train, y_train,\n",
    "              batch_size=batch_size,\n",
    "              epochs=epochs,\n",
    "              validation_data=(x_test, y_test),\n",
    "              shuffle=True,\n",
    "              callbacks=[csv_logger, checkpointer, early_stopper]) \n",
    "else:\n",
    "    print('Using real-time data augmentation.')\n",
    "    # This will do preprocessing and realtime data augmentation:\n",
    "    datagen = ImageDataGenerator(\n",
    "        featurewise_center=False,  # set input mean to 0 over the dataset\n",
    "        samplewise_center=False,  # set each sample mean to 0\n",
    "        featurewise_std_normalization=False,  # divide inputs by std\n",
    "        samplewise_std_normalization=False,  # divide each input by its std\n",
    "        zca_whitening=False,  # apply ZCA whitening\n",
    "        # randomly rotate images in the range (degrees, 0 to 180)\n",
    "        rotation_range=0,\n",
    "        # randomly shift images horizontally (fraction of total width)\n",
    "        width_shift_range=0.1,\n",
    "        # randomly shift images vertically (fraction of total height)\n",
    "        height_shift_range=0.1,\n",
    "        horizontal_flip=True,  # randomly flip images\n",
    "        vertical_flip=False)  # randomly flip images\n",
    "\n",
    "    # Compute quantities required for feature-wise normalization\n",
    "    # (std, mean, and principal components if ZCA whitening is applied).\n",
    "    datagen.fit(x_train)\n",
    "\n",
    "    # Fit the model on the batches generated by datagen.flow().\n",
    "    model.fit_generator(datagen.flow(x_train, y_train,\n",
    "                                     batch_size=batch_size),\n",
    "                        steps_per_epoch=x_train.shape[0] // batch_size,\n",
    "                        epochs=epochs,\n",
    "                        validation_data=(x_test, y_test),\n",
    "                        callbacks=[csv_logger, checkpointer, \n",
    "                                   early_stopper, lr_reducer,\n",
    "                                   TQDMNotebookCallback()])\n",
    "    model.save('{}.h5'.format(title))\n",
    "    \n",
    "    # Predict\n",
    "    y_train_pred_enc = model.predict(x_train)\n",
    "    y_test_pred_enc = model.predict(x_test)\n",
    "    \n",
    "    # Save labels, predictions\n",
    "    from pulearn.utils import synthesize_pu_labels\n",
    "    pct_missing = 0.5\n",
    "    y_train_pu = synthesize_pu_labels(y_train, random_state=42, verbose=True)\n",
    "    \n",
    "    lst = []\n",
    "    y_train_pred = np.argmax(y_train_pred_enc, axis=-1)\n",
    "    y_train_true = np.argmax(y_train, axis=-1)\n",
    "    y_train_label = np.argmax(y_train_pu[pct_missing], axis=-1)\n",
    "    for y_pred, y_true, y_label in zip(y_train_pred, y_train_true, y_train_label):\n",
    "        lst.append(dict(y_pred=y_pred, y_true=y_true, y_label=y_label, split='train'))\n",
    "\n",
    "    y_test_pred = np.argmax(y_test_pred_enc, axis=-1)\n",
    "    y_test_true = np.argmax(y_test, axis=-1)\n",
    "    for y_pred, y_true in zip(y_test_pred, y_test_true):\n",
    "        lst.append(dict(y_pred=y_pred, y_true=y_true, y_label=y_true, split='test'))\n",
    "    \n",
    "    df = pd.DataFrame(lst)\n",
    "    df.to_csv('prediction_{}.csv'.format(title), index=False, mode='a')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
